﻿<?php
ini_set('date.timezone','Asia/Shanghai');
include 'simple_html_dom.php';
include 'config_catchbaidu.php';

define('APPS_NUM',2405);
define('GAMES_NUM',977);

define('DBPAS',"coolchuan2012");
//define('DBPAS',"123456");

$link = mysql_connect("localhost","root",DBPAS); 
mysql_select_db("catchappdataall", $link);
mysql_query('SET NAMES UTF8');


define('ONCECATCHCOUNT',30);
define('ONCECOMMENTCATCHCOUNT',50);

define('WEBURLHEADER','http://as.baidu.com/a/item?docid=');

define('LOGPATHHEADER','/var/www/catchjiankongdata/');
//define('LOGPATHHEADER','');

define('LOGFORSUCCESS',0);

define('WAPURLHEADERBAIDU','http://as.baidu.com/a/item?docid=');
define('WAPURLFOOTERBAIDU','&pre=web_am_software&pos=software_1012_0&f=suite_3%40software_1012_0&suite=3&bdtype=soft&appname=soft&version=2.2');
define('WEBURLHEADERBAIDU','http://as.baidu.com/a/item?docid=');


catchother();
//getinfoandrank();

//getcomment();


function getinfoandrank()
{
	//获取已存在app基础数据
//	catchother();
	//应用分类
	$apptype_arr = getapptypelist();
	foreach ($apptype_arr as $apptype)
	{
		if(($apptype['id']>=1) && ($apptype['id']<=5))
		{
			//每一类的url
			$apps_url_app_type_list = $apptype['url'];
			decodeurlandgetdata($apptype['id'],"soft",$apps_url_app_type_list);
		}
	}
	//应用总榜
	decodeurlandgetdata(101,"soft","http://as.baidu.com/a/software?cid=101&s=1&f=suite_3&suite=3&bdtype=soft&appname=soft&version=2.2&pn=");
	
	
//	游戏分类
//	$gametype_arr = getgametypelist();
//	foreach ($gametype_arr as $gametype)
//	{
//		每一类的url
//		$apps_url_game_type_list = $gametype['url'];
//		decodeurlandgetdata($gametype['id'],"game",$apps_url_game_type_list);
//	}
//	游戏总榜
//	decodeurlandgetdata(102,"game","http://as.baidu.com/a/asgame?cid=102&s=1&f=suite_3&suite=3&bdtype=soft&appname=soft&version=2.2&pn=");
	catchother();
}



function decodeurlandgetdata($catchtype,$appbigtype,$urlheader)
{
	$app_exit_infos = get_exit_appinfo_fromdb();
	$app_exit_versions = get_exit_appversion_fromdb();
	
	$app_today_data = get_today_data();
	$app_today_score = get_today_score();
	
	for($i=1;;)
	{
		$apps_url_app_all_list = $urlheader.$i;
	    
		//有内容，解析
		$rankpos = ($i-1) * ONCECATCHCOUNT;
		$app_info_arr_insertdb_arr = catchinfodata($catchtype,$appbigtype,$rankpos,$apps_url_app_all_list);
		
		if($i >= 68)
		{
			break;
		}
		
		$app_info_arr_insertdb_data = array();
		$app_info_arr_insertdb_scoredata = array();
		//写数据库
		foreach ($app_info_arr_insertdb_arr as $app_info_arr_insertdb)
		{
			//判断是否需要更新信息库
			$searchinfoid = searchisexit($app_info_arr_insertdb, $app_exit_infos);
			if($searchinfoid == 0)
			{
				$app_info_arr_insertdb['app_id'] = insert_appinfo_ifnotexit_todb($app_info_arr_insertdb);
				$app_info_arr_insertdb['app_version_id'] = insert_appversion_ifnotexit_todb($app_info_arr_insertdb);
				
				//重新获取存在app信息，因为有了更新
				$app_exit_infos = get_exit_appinfo_fromdb();
				$app_exit_versions = get_exit_appversion_fromdb();
			}
			else 
			{
				$app_info_arr_insertdb['app_id'] = $searchinfoid;
				$searchversionid = searchisexitversion($app_info_arr_insertdb, $app_exit_versions);
				if($searchversionid == 0)
				{
					$app_info_arr_insertdb['app_version_id'] = insert_appversion_ifnotexit_todb($app_info_arr_insertdb);
					
					//重新获取存在app信息，因为有了更新
					$app_exit_versions = get_exit_appversion_fromdb();
				}
				else
				{
					$app_info_arr_insertdb['app_version_id'] = $searchversionid;
				}
			}
			
			//判断当天没有抓过，再写入
			if(searchisexittodaydata($app_info_arr_insertdb, $app_today_data) == 0)
			{
				//update_appinfo_todb($app_info_arr_insertdb);
				//$app_today_data = get_today_data();
				$app_info_arr_insertdb_data[] = $app_info_arr_insertdb;
			}
			if(searchisexittodayscore($app_info_arr_insertdb, $app_today_score) == 0)
			{
				//update_appscore_todb($app_info_arr_insertdb);
				//$app_today_score = get_today_score();
				$app_info_arr_insertdb_scoredata[] = $app_info_arr_insertdb;
			}
		}
		
		update_appinfo_todb($app_info_arr_insertdb_data);
		update_appscore_todb($app_info_arr_insertdb_scoredata);
//		$app_today_data = get_today_data();
//		$app_today_score = get_today_score();
		$app_info_arr_insertdb_data = array();
		$app_info_arr_insertdb_scoredata = array();
		
		$i++;
		
	//	结束
	//	if($obj->total < ONCECATCHCOUNT)
	//	{
	//		break;
	//	}
	}
}


function searchisexit($app_infos, $app_exit_info) 
{
	foreach ($app_exit_info as $app_exit) 
	{
		if ($app_infos['packagename'] == $app_exit["AppPackageName"]) 
		{
			if(($app_infos['AppID']>0) && ($app_infos['AppID']>$app_exit['app_id']))
			{
				$link = mysql_connect("localhost","root",DBPAS); 
	
				$sql = "update jk_apps_info_baidu set app_id = ".$app_infos['AppID']." where id = ".$app_exit['id']." and app_id = ".$app_exit['app_id']." ";
				$rs = mysql_query($sql, $link);
			}
			return $app_exit['id'];
		}
	}
	return 0;
}

function searchisexitversion($app_infos, $app_exit_version) 
{
	foreach ($app_exit_version as $app_exit) 
	{
		if (($app_infos['app_id'] == $app_exit["app_id"]) && ($app_infos['AppVersion'] == $app_exit["versionName"]) ) 
		{
			return $app_exit['id'];
		}
	}
	return 0;
}

function searchisexittodaydata($app_infos, $app_today_data) 
{
	foreach ($app_today_data as $app_exit) 
	{
		if (($app_infos['app_id'] == $app_exit["app_id"]) && ($app_infos['AppRanking'] == $app_exit["rank"]) && ($app_infos['AppRankingType'] == $app_exit["ranktype"]))
		{
			return $app_exit['id'];
		}
	}
	return 0;
}
function searchisexittodayscore($app_infos, $app_today_score) 
{
	foreach ($app_today_score as $app_exit)
	{
		if (($app_infos['app_id'] == $app_exit["app_id"]))
		{
			return $app_exit['id'];
		}
	}
	return 0;
}


//排名分类，app大类，app信息
function catchinfodata($catchtype,$appbigtype,$flag,$apps_url_app_all_list)
{
	$ranking = $flag ;//* ONCECATCHCOUNT;
	$app_info_insertdb_arr = array();
	
	// 新建一个Dom实例 
	$html = new simple_html_dom();
	// 从url中加载 
	@$html = file_get_html($apps_url_app_all_list);
	if($html == "")
	{
		@$html = file_get_html($apps_url_app_all_list);
		if($html == "")
		{
			return $app_info_insertdb_arr;
		}
	}
	
	
	$app_url_arr = array();
	$ranking = $flag ;
	$app_info_insertdb_arr = array();
	

	$appdata_arr = $html->find('div.filter-app-list-wrapper ul.filter-app-list li');
	foreach($appdata_arr as $appdata_temp) 
	{
		$appdata = $appdata_temp;
		
		$ranking++;
		$app_info_insertdb = array();

		$appinfotemp = $appdata->find('div.normal-show div.pic a img',0);
		
		$app_info_insertdb['AppLogo'] = $appinfotemp->src;
		
		$appinfotemp = $appdata->find('div.normal-show h4 a',0);
		$wapurl = $appinfotemp->href;
		$app_info_insertdb['name'] = addslashes($appinfotemp->plaintext);
		
		//解析属性
		$appinfotemp = $appdata->find('div.normal-show div.down-num a',0);
		
		$appinfostr = $appinfotemp->outertext;
		$app_info_insertdb['AppVersion'] = getdatafromstring($appinfostr,"data-versionname");
		$app_info_insertdb['Versioncode'] = getdatafromstring($appinfostr,"data-versioncode");
		$app_info_insertdb['packagename'] = getdatafromstring($appinfostr,"data-package");
		$app_info_insertdb['AppID'] = getdatafromstring($appinfostr,"data-docid");
		$app_info_insertdb['AppSize'] = getdatafromstring($appinfostr,"data-size");

		$app_info_insertdb['url'] = WEBURLHEADER.$app_info_insertdb['AppID'];
		
		$appinfo_arr_temp = getappdetailinfo($wapurl);
		
		//使用底层的title，列表title有省略
		$app_info_insertdb['name'] = $appinfo_arr_temp['name'];
		$app_info_insertdb['AppDownTimes'] = $appinfo_arr_temp['downtimes'];
		$app_info_insertdb['Rating'] = $appinfo_arr_temp['Rating'];
		
		$app_info_insertdb['commentid'] = $appinfo_arr_temp['commentid'];
		
		$app_info_insertdb['CommentCount'] = $appinfo_arr_temp['commentcount'];
		
		
		$app_info_insertdb['AppType'] = "";
//		if($catchtype != 100)
		{
			$app_info_insertdb['AppType'] = $catchtype;
		}
		$app_info_insertdb['AppBigType'] = $appbigtype;
		$app_info_insertdb['AppRanking'] = $ranking;
		$app_info_insertdb['AppRankingType'] = $catchtype;
		$app_info_insertdb_arr[] = $app_info_insertdb;
	}
	$html->clear();
	return $app_info_insertdb_arr;
}

function getdatafromstring($appinfostr,$tagname)
{
	$pos = strpos($appinfostr, $tagname);
	$appinfostr_tmp = substr($appinfostr,$pos,(strlen($appinfostr)-$pos));
	$pos = strpos($appinfostr_tmp, "\"");
	$appinfostr_tmp = substr($appinfostr_tmp,$pos+1,(strlen($appinfostr)-$pos));
	$pos = strpos($appinfostr_tmp, "\"");
	$appinfostr_tmp = substr($appinfostr_tmp,0,$pos);
	return $appinfostr_tmp;
}
function getappdetailinfo($app_url)
{
	$appdetailinfo_arr = array();
	
	// 新建一个Dom实例 
	$html_detail = new simple_html_dom();
	// 从url中加载 
	@$html_detail = file_get_html($app_url);
	if($html_detail == "")
	{
		@$html_detail = file_get_html($app_url);
		if($html_detail == "")
		{
			return $appdetailinfo_arr;
		}
	}
	
	$appname = $html_detail->find('div.info-top dl.clearfix span[id=appname]',0)->plaintext;
	$appdetailinfo_arr['name'] = addslashes($appname);

	$scoredata = $html_detail->find('div.info-top dl.clearfix dd.info-star b.color-orange',0)->plaintext;
	$scoredata = str_replace("分","",$scoredata);
	$scoredata = str_replace(" ","",$scoredata);
	$appdetailinfo_arr['Rating'] = $scoredata;
	
	$appdetailinfo = $html_detail->find('div.info-top dl.clearfix dd.info-params span.params-download-num',0);

	$appdetailinfo_arr['downtimes'] = $appdetailinfo->plaintext;
	if(strstr($appdetailinfo_arr['downtimes'],"+"))
	{
		$appdetailinfo_arr['downtimes'] = str_replace("+","",$appdetailinfo_arr['downtimes']);
		$appdetailinfo_arr['downtimes'] = str_replace(" ","",$appdetailinfo_arr['downtimes']);
	}
	
	$commentidstr = $html_detail->find('div.middle div.cls div.content-main div.content-main-border section.s-index-page',0)->outertext;
	$appdetailinfo_arr['commentid'] = getdatafromstring($commentidstr,"data-groupid");
	
	//评分人数
	$scoredata = $html_detail->find('div.info-top dl.clearfix dd.info-star span.color-gray',0)->plaintext;
	$scoredata = str_replace("已有","",$scoredata);
	$scoredata = str_replace("人评分","",$scoredata);
	$scoredata = str_replace("(","",$scoredata);
	$scoredata = str_replace(")","",$scoredata);
	$scoredata = str_replace(" ","",$scoredata);
	$appdetailinfo_arr['commentcount'] = $scoredata;
	
	
	$html_detail->clear();
	return $appdetailinfo_arr;
}



function get_exit_appinfo_fromdb()
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));

	$sql = 'select id, app_id, packageName,title,apptype,appbigtype from jk_apps_info_baidu where IsDeleted=0 '; 
	$rs = mysql_query($sql, $link);
	
	$app_exit_info = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_info_temp = array();
  		$app_exit_info_temp['id'] = $row['id'];
  		$app_exit_info_temp['app_id'] = $row['app_id'];
	  	$app_exit_info_temp['AppPackageName'] = $row['packageName'];
	  	$app_exit_info_temp['AppName'] = $row['title'];
	  	$app_exit_info_temp['apptype'] = $row['apptype'];
	  	$app_exit_info_temp['appbigtype'] = $row['appbigtype'];
	  	$app_exit_info[] = $app_exit_info_temp;
  	}
	return $app_exit_info;
}

function get_exit_appversion_fromdb()
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));

	$sql = 'select id,app_id,versionName,versionCode from jk_apps_version_baidu where IsDeleted=0 '; 
	$rs = mysql_query($sql, $link);
	
	$app_exit_version = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_version_temp = array();
	  	$app_exit_version_temp['id'] = $row['id'];
	  	$app_exit_version_temp['app_id'] = $row['app_id'];
	  	$app_exit_version_temp['versionName'] = $row['versionName'];
	  	$app_exit_version_temp['versionCode'] = $row['versionCode'];
	  	$app_exit_version[] = $app_exit_version_temp;
  	}
	return $app_exit_version;
}

function get_today_data()
{
	$today = date("Y-m-d");
	$link = mysql_connect("localhost","root",DBPAS); 
	
	$sql = 'select id, app_id,rank,ranktype,time from jk_apps_data_mobile_baidu_'.date('Ym',time()).' where IsDeleted=0 and time="'.$today.'"'; 
	$rs = mysql_query($sql, $link);
	
	$app_exit_info = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_info_temp = array();
  		$app_exit_info_temp['id'] = $row['id'];
  		$app_exit_info_temp['app_id'] = $row['app_id'];
  		$app_exit_info_temp['rank'] = $row['rank'];
	  	$app_exit_info_temp['ranktype'] = $row['ranktype'];
	  	$app_exit_info_temp['time'] = $row['time'];
	  	$app_exit_info[] = $app_exit_info_temp;
  	}
	return $app_exit_info;
}
function get_today_score()
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$today = date("Y-m-d");

	$sql = 'select id, app_id from jk_apps_score_data_mobile_baidu_'.date('Ym',time()).' where IsDeleted=0 and time="'.$today.'"'; 
	$rs = mysql_query($sql, $link);
	
	$app_exit_info = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_info_temp = array();
  		$app_exit_info_temp['id'] = $row['id'];
	  	$app_exit_info_temp['app_id'] = $row['app_id'];
	  	$app_exit_info[] = $app_exit_info_temp;
  	}
	return $app_exit_info;
}

//app基础信息
function insert_appinfo_ifnotexit_todb($app_info)
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));

	$sql = 'insert into jk_apps_info_baidu (app_id,title,packageName,url,icon,apptype,appbigtype)
	 values ('.$app_info['AppID'].',"'.$app_info['name'].'","'.$app_info['packagename'].'","'.$app_info['url'].'","'.$app_info['AppLogo'].'","'.$app_info['AppType'].'","'.$app_info['AppBigType'].'")'; 

	$rs = mysql_query($sql, $link);
	//echo $sql;die();
	$filename_success = LOGPATHHEADER.'logs/baidu_info_success.log';
	$filename_fail = LOGPATHHEADER.'logs/baidu_info_fail.log';
	$filename_db_fail = LOGPATHHEADER.'logs/baidu_info_db_fail.log';

	try
	{
		if(!$rs)
		{
			//记录错误，二次抓取用
			$info_str = $app_info["name"].$sql."Valid result!\r\n";
			$fh = fopen($filename_db_fail, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
		}
		else
		{
			if(LOGFORSUCCESS)
			{
				$info_str = $app_info["name"]."success!\r\n";
				$filename = $filename_success;
				$fh = fopen($filename, "a");
				fwrite($fh, $info_str);
				fclose($fh);
			}
		}
	}
	catch(Exception $e)
	{
		
	}
	
	return mysql_insert_id();
}
//app version信息
function insert_appversion_ifnotexit_todb($app_info)
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));

	$sql = 'insert into jk_apps_version_baidu (app_id,versionName,versionCode)
	 values ('.$app_info['app_id'].',"'.$app_info['AppVersion'].'","'.$app_info['Versioncode'].'")';

	$rs = mysql_query($sql, $link);
	//echo $sql;die();
	$filename_success = LOGPATHHEADER.'logs/baidu_info_success.log';
	$filename_fail = LOGPATHHEADER.'logs/baidu_info_fail.log';
	$filename_db_fail = LOGPATHHEADER.'logs/baidu_info_db_fail.log';

	try
	{
		if(!$rs)
		{
			//记录错误，二次抓取用
			$info_str = $app_info["name"].$sql."Valid result!\r\n";
			$fh = fopen($filename_db_fail, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
		}
		else
		{
			if(LOGFORSUCCESS)
			{
				$info_str = $app_info["name"]."success!\r\n";
				$filename = $filename_success;
				$fh = fopen($filename, "a");
				fwrite($fh, $info_str);
				fclose($fh);
			}
		}
	}
	catch(Exception $e)
	{
		
	}
	return mysql_insert_id();
}
//排名信息
function update_appinfo_todb($app_info_arr)
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));
	
	$today = date("Y-m-d");
	
	if(count($app_info_arr) == 0)
	{
		return ;
	}
	
	$sql = 'insert into jk_apps_data_mobile_baidu_'.date('Ym',time()).' (app_id,app_version_id,rank,ranktype,time) values '; 

	$i=0;
	foreach ($app_info_arr as $app_info)
	{
		if($i==0)
		{
			$sql .= '('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppRanking'].','.$app_info['AppRankingType'].',"'.$today.'")';
		}
		else
		{
			$sql .= ',('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppRanking'].','.$app_info['AppRankingType'].',"'.$today.'")';
		}
		$i++;
	}
	
//	$sql = 'insert into jk_apps_data_mobile_baidu_'.date('Ym',time()).' (app_id,app_version_id,rank,ranktype,time) values ('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppRanking'].','.$app_info['AppRankingType'].',"'.$today.'")'; 

//	echo $sql;die();
	$rs = mysql_query($sql, $link);
	
	$filename_success = LOGPATHHEADER.'logs/baidu_success.log';
	$filename_fail = LOGPATHHEADER.'logs/baidu_fail.log';
	$filename_db_fail = LOGPATHHEADER.'logs/baidu_db_fail.log';

	try
	{
		if(!$rs)
		{
			//记录错误，二次抓取用
			$info_str = $sql."Valid result!\r\n";
			$fh = fopen($filename_db_fail, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
		}
		else
		{
			if(LOGFORSUCCESS)
			{
				$info_str = $app_info["name"]."success!\r\n";
				$filename = $filename_success;
				$fh = fopen($filename, "a");
				fwrite($fh, $info_str);
				fclose($fh);
			}
		}
	}
	catch(Exception $e)
	{
		
	}
}


function getcommentcount($commentid)
{
	//获取已存在app基础数据
	$apps_url_comment = "http://as.baidu.com/a/comment?action_type=getCommentList&groupid=$commentid&pn=1";
	
	@$json_string = file_get_html($apps_url_comment);
	$obj = json_decode($json_string);
	if($obj->total_count != null)
	{
		return  $obj->total_count;
	}
	return  0;
}



//监控信息
function update_appscore_todb($app_info_arr)
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));
	
	$today = date("Y-m-d");
	if(count($app_info_arr) == 0)
	{
		return ;
	}
	
	$sql = 'insert into jk_apps_score_data_mobile_baidu_'.date('Ym',time()).' (app_id,app_version_id,downloads,score,scoretype,comments,time) values '; 

	$i=0;
	foreach ($app_info_arr as $app_info)
	{
		if(strstr($app_info['AppDownTimes'],"亿"))
		{
			$app_info['AppDownTimes'] = str_replace("亿","",$app_info['AppDownTimes']) * 100000000;
		}
		if(strstr($app_info['AppDownTimes'],"万"))
		{
			$app_info['AppDownTimes'] = str_replace("万","",$app_info['AppDownTimes']) * 10000;
		}
		if(strstr($app_info['AppDownTimes'],"千"))
		{
			$app_info['AppDownTimes'] = str_replace("千","",$app_info['AppDownTimes']) * 1000;
		}
		if(strstr($app_info['AppDownTimes'],"百"))
		{
			$app_info['AppDownTimes'] = str_replace("百","",$app_info['AppDownTimes']) * 100;
		}
		if($app_info['AppDownTimes'] == "")
		{
			$app_info['AppDownTimes'] = 0;
		}
		if($app_info['Rating'] == "")
		{
			$app_info['Rating'] = 0;
		}
		
		$commentcount = getcommentcount($app_info['commentid']);
//		$commentcount = $app_info['CommentCount'];
		if(!($commentcount > 0 ))
		{
			$commentcount = 0;
		}
		
		if($commentcount == "")
		{
			$commentcount = 0;
		}
		
		if($i==0)
		{
			$sql .= ' ('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppDownTimes'].','.$app_info['Rating'].',"",'.$commentcount.',"'.$today.'")';
		}
		else
		{
			$sql .= ', ('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppDownTimes'].','.$app_info['Rating'].',"",'.$commentcount.',"'.$today.'")';
		}
		$i++;
	}
	
	
	
//	$sql = 'insert into jk_apps_score_data_mobile_baidu_'.date('Ym',time()).' (app_id,app_version_id,downloads,score,scoretype,comments,time) values ('.$app_info['app_id'].','.$app_info['app_version_id'].','.$app_info['AppDownTimes'].','.$app_info['Rating'].',"",'.$commentcount.',"'.$today.'")'; 

//	echo $sql;die();
	$rs = mysql_query($sql, $link);
	
	$filename_success = LOGPATHHEADER.'logs/baidu_success.log';
	$filename_fail = LOGPATHHEADER.'logs/baidu_fail.log';
	$filename_db_fail = LOGPATHHEADER.'logs/baidu_db_fail.log';
	$filename_nopage = LOGPATHHEADER.'logs/baidu_nopage.log';
	$filename_erroe_packagename = LOGPATHHEADER.'logs/baidu_error_packagename.log';
	try
	{
		if(!$rs)
		{
			//记录错误，二次抓取用
			$info_str = $app_info['app_id']."\r\n";
			$fh = fopen($filename_nopage, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
			//记录错误，二次抓取用
			$info_str = $app_info['packagename']."\r\n";
			$fh = fopen($filename_erroe_packagename, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
			//记录错误，二次抓取用
			$info_str = $sql."Valid result!\r\n";
			$fh = fopen($filename_db_fail, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
		}
		else
		{
			if(LOGFORSUCCESS)
			{
				$info_str = $app_info["name"]."success!\r\n";
				$filename = $filename_success;
				$fh = fopen($filename, "a");
				fwrite($fh, $info_str);
				fclose($fh);
			}
		}
	}
	catch(Exception $e)
	{
		
	}
}





//评论
function getcomment()
{
	//获取已存在app基础数据
	
	$app_exit_infos = get_exit_appinfo_fromdb();
	foreach ($app_exit_infos as $app_exit)
	{
		$appname =  urlencode($app_exit['AppName']);
		
		decodeurlandgetcomment($app_exit,"http://openbox.comment.mapp.360.cn/message/getmessage/type/best|normal|bad/name/".$appname."/start/");
	}
}

function decodeurlandgetcomment($app_exit,$urlheader)
{
	$app_exit_infos = get_exit_appinfo_fromdb();
	$app_exit_versions = get_exit_appversion_fromdb();
	$app_exit_comment = get_exit_appcomment_fromdb();
	
	for($i=0;;)
	{
		$apps_url_comment = $urlheader.$i."/count/".ONCECOMMENTCATCHCOUNT;
		
		@$json_string = file_get_html($apps_url_comment);
		$obj = json_decode($json_string);
		
		$appinfoarr = $obj->data->messages;
		//$obj->data->total
		
		//本次为空，结束
		if((count($appinfoarr) == 0) || ($appinfoarr == ""))
		{
			break;
		}
		
		//有内容，解析
		$app_info_arr_insertdb_arr = catchcommentdata($appinfoarr);
		
		
		//写数据库
		foreach ($app_info_arr_insertdb_arr as $app_info_arr_insertdb)
		{
			$app_info_arr_insertdb['app_id'] = $app_exit['id'];
			$app_info_arr_insertdb['app_version_id'] = 0;
			foreach ($app_exit_versions as $app_versions)
			{
				if($app_versions['app_id'] == $app_info_arr_insertdb['app_id'])
				{
					$app_info_arr_insertdb['app_version_id'] = $app_versions['id'];
					break;
				}
			}
			//判断是否需要更新信息库
			$searchcommentid = searchcommentisexit($app_info_arr_insertdb, $app_exit_comment);
			if($searchcommentid == 0)
			{
				update_appcomment_todb($app_info_arr_insertdb,0);
				//_ifnotexit_todb($app_info_arr_insertdb);
				
				//重新获取存在app信息，因为有了更新
				$app_exit_comment = get_exit_appcomment_fromdb();
			}
		}
		$i = $i + ONCECOMMENTCATCHCOUNT;
	}
}

//排名分类，app大类，app信息
function catchcommentdata($appcommentarr)
{
	$app_info_insertdb_arr = array();
	
	foreach ($appcommentarr as $appcomment)
	{
		$app_info_insertdb = array();
		
//		$app_info_insertdb['id'] = $appcomment->id;
		$app_info_insertdb['content'] = addslashes($appcomment->content);
		$app_info_insertdb['date'] = $appcomment->create_time;
		$app_info_insertdb['author_name'] = addslashes($appcomment->username);
		$app_info_insertdb['author_id'] = "";//$appcomment->authorId;
		$app_info_insertdb['type'] = $appcomment->type;
		$app_info_insertdb_arr[] = $app_info_insertdb;
	}
	return $app_info_insertdb_arr;
}

//监控信息
function update_appcomment_todb($app_comment,$platform_id)
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));
	
	//$datano = date("Y-m-d",strtotime($app_comment["date"]));
	
	$sql = 'insert into jk_apps_comments_detail_baidu (app_id,app_version_id,platform_id,author_id,author_name,content,scoretype,time,datetime)
	 values ('.$app_comment['app_id'].','.$app_comment['app_version_id'].','.$platform_id.',"'.$app_comment['author_id'].'","'.$app_comment['author_name'].'","'.$app_comment['content'].'","'.$app_comment['type'].'","'.$app_comment['date'].'","'.$app_comment['date'].'")'; 

//	echo $sql;die();
	$rs = mysql_query($sql, $link);
	
	$filename_success = LOGPATHHEADER.'logs/baidu_success.log';
	$filename_fail = LOGPATHHEADER.'logs/baidu_fail.log';
	$filename_db_fail = LOGPATHHEADER.'logs/baidu_db_fail.log';

	try
	{
		if(!$rs)
		{
			//记录错误，二次抓取用
			$info_str = $app_comment["app_id"].$sql."Valid result!\r\n";
			$fh = fopen($filename_db_fail, "a");
			fwrite($fh, $info_str);
			fclose($fh);
			echo $info_str;
		}
		else
		{
			if(LOGFORSUCCESS)
			{
				$info_str = $app_comment["app_id"]."success!\r\n";
				$filename = $filename_success;
				$fh = fopen($filename, "a");
				fwrite($fh, $info_str);
				fclose($fh);
			}
		}
	}
	catch(Exception $e)
	{
		
	}
}



function get_exit_appcomment_fromdb()
{
	$link = mysql_connect("localhost","root",DBPAS); 
	
	$sql = 'select id,app_id,app_version_id,author_name,content,time,datetime from jk_apps_comments_detail_baidu where IsDeleted=0 '; 
	$rs = mysql_query($sql, $link);
	
	$app_exit_comment = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_comment_temp = array();
	  	$app_exit_comment_temp['id'] = $row['id'];
	  	$app_exit_comment_temp['app_id'] = $row['app_id'];
	  	$app_exit_comment_temp['app_version_id'] = $row['app_version_id'];
	  	$app_exit_comment_temp['author_name'] = $row['author_name'];
	  	$app_exit_comment_temp['content'] = $row['content'];
	  	$app_exit_comment_temp['time'] = $row['time'];
	  	$app_exit_comment_temp['datetime'] = $row['datetime'];
	  	$app_exit_comment[] = $app_exit_comment_temp;
  	}
	return $app_exit_comment;
}

function searchcommentisexit($app_comment, $app_exit_comment)
{
	foreach ($app_exit_comment as $app_exit) 
	{
		if (($app_comment['app_id'] == $app_exit["app_id"]) && ($app_comment['app_version_id'] == $app_exit["app_version_id"]) && ($app_comment['author_name'] == $app_exit["author_name"]) && ($app_comment['content'] == $app_exit["content"]) && ($app_comment['date'] == $app_exit["datetime"])) 
		{
			return $app_exit['id'];
		}
	}
	return 0;
}



function catchother()
{
	$app_exit_infos = get_exit_appinfo_fromdb_forcatch();
	$app_exit_versions = get_exit_appversion_fromdb();
	
	$app_today_data = get_today_data();
	$app_today_score = get_today_score();
	
	$app_today_score_id_arr = array();
	foreach ($app_today_score as $app_today_score_data)
	{
		$app_today_score_id_arr[] = $app_today_score_data["app_id"];
	}
	
	foreach ($app_exit_infos as $app_exit_info)
	{
		writelog(1,$app_exit_info['id']);
		if(in_array($app_exit_info['id'],$app_today_score_id_arr))
		{//已抓
			continue;
		}
		writelog(0,$app_exit_info['id']);
		//未抓
		$packagename = $app_exit_info['AppPackageName'];
		if($packagename == "")
		{
			continue;
		}
		
		$idbaidu = $app_exit_info['app_id'];
		if($idbaidu == "")
		{
			continue;
		}
		
		$wapurl = WAPURLHEADERBAIDU.$idbaidu.WAPURLFOOTERBAIDU;
	
//		$appdetail_info_arr = getappdetailinfobaidu($packagename,$idbaidu,$wapurl,$app_exit_info['apptype'],$app_exit_info['appbigtype']);
		
		
		$app_info_arr_insertdb = getappdetailinfobaidu($packagename,$idbaidu,$wapurl,$app_exit_info['apptype'],$app_exit_info['appbigtype']);
		
		if(count($app_info_arr_insertdb) == 0)
		{
			continue;
		}
		
		$app_info_arr_insertdb['AppRanking'] = 0;//这里抓的没有排名
		$app_info_arr_insertdb['AppRankingType'] = $app_exit_info['apptype'];
		
		$app_info_arr_insertdb['app_id'] = 0;
		$app_info_arr_insertdb['app_version_id'] = 0;
		
		//判断是否需要更新信息库
		$searchinfoid = searchisexit($app_info_arr_insertdb, $app_exit_infos);
		if($searchinfoid == 0)
		{
			//理论上不可能不存在，因为是从这个数组中取的
			
//			$app_info_arr_insertdb['app_id'] = insert_appinfo_ifnotexit_todb($app_info_arr_insertdb);
//			$app_info_arr_insertdb['app_version_id'] = insert_appversion_ifnotexit_todb($app_info_arr_insertdb);
//			
//			//重新获取存在app信息，因为有了更新
//			$app_exit_infos = get_exit_appinfo_fromdb();
//			$app_exit_versions = get_exit_appversion_fromdb();
		}
		else 
		{
			$app_info_arr_insertdb['app_id'] = $searchinfoid;
			$searchversionid = searchisexitversion($app_info_arr_insertdb, $app_exit_versions);
			if($searchversionid == 0)
			{
				//理论上不可能不存在，因为是从这个数组中取的
				
//				$app_info_arr_insertdb['app_version_id'] = insert_appversion_ifnotexit_todb($app_info_arr_insertdb);
//				
//				//重新获取存在app信息，因为有了更新
//				$app_exit_versions = get_exit_appversion_fromdb();
			}
			else 
			{
				$app_info_arr_insertdb['app_version_id'] = $searchversionid;
			}
		}
		
		//判断当天没有抓过，再写入
		if(searchisexittodaydata($app_info_arr_insertdb, $app_today_data) == 0)
		{
			//update_appinfo_todb($app_info_arr_insertdb);
			//$app_today_data = get_today_data();
			$app_info_arr_insertdb_data[] = $app_info_arr_insertdb;
		}
		if(searchisexittodayscore($app_info_arr_insertdb, $app_today_score) == 0)
		{
			//update_appscore_todb($app_info_arr_insertdb);
			//$app_today_score = get_today_score();
			$app_info_arr_insertdb_scoredata[] = $app_info_arr_insertdb;
		}
	
		//此处的都是排行榜之外的，因此不写排行榜之外的数据
	//		update_appinfo_todb($app_info_arr_insertdb_data);
		update_appscore_todb($app_info_arr_insertdb_scoredata);
	//		$app_today_data = get_today_data();
	//		$app_today_score = get_today_score();
		$app_info_arr_insertdb_data = array();
		$app_info_arr_insertdb_scoredata = array();
	}
}

function getappdetailinfobaidu($packagename,$idbaidu,$app_detail_url,$catchtype,$appbigtype)
{
	$appdetailinfo_arr = array();
	
	// 新建一个Dom实例 
	$html_detail = new simple_html_dom();
	// 从url中加载 
	@$html_detail = file_get_html($app_detail_url);
	if($html_detail == "")
	{
		@$html_detail = file_get_html($app_detail_url);
		if($html_detail == "")
		{
			return $appdetailinfo_arr;
		}
	}
	
	$appinfo_temp = $html_detail->find('div.info-top dl.clearfix',0);
	
	if($appinfo_temp == "")
	{
		writefailelog(1,$packagename);
		return $appdetailinfo_arr;
	}
	
	$appdetailinfo_arr['AppLogo'] = $appinfo_temp->find('dt.fleft img[id=app-logo]',0)->src;
	//解析属性
	$appinfo_temp_param = $appinfo_temp->find('dd.info-params table tr',0);
	$appdetailinfo_arr['AppVersion'] = $appinfo_temp_param->find('span.params-vname',0)->plaintext;
	$appdetailinfo_arr['Versioncode'] = "";
	$appdetailinfo_arr['packagename'] = $packagename;
	$appdetailinfo_arr['AppID'] = $idbaidu;
	$appdetailinfo_arr['AppSize'] = $appinfo_temp_param->find('span.params-size',0)->plaintext;
	$appdetailinfo_arr['url'] = WEBURLHEADERBAIDU.$appdetailinfo_arr['AppID'];
	
	$appdetailinfo_arr['AppDownTimes'] = $appinfo_temp->find('dd.info-params table tr',1)->find('span.params-download-num',0)->plaintext;
	if(strstr($appdetailinfo_arr['AppDownTimes'],"+"))
	{
		$appdetailinfo_arr['AppDownTimes'] = str_replace("+","",$appdetailinfo_arr['AppDownTimes']);
		$appdetailinfo_arr['AppDownTimes'] = str_replace(" ","",$appdetailinfo_arr['AppDownTimes']);
	}
	$appdetailinfo_arr['AppDownTimes'] = getdownloadtimes($appdetailinfo_arr['AppDownTimes']);
	
	$appdetailinfo_arr['AppType'] = $catchtype;
	$appdetailinfo_arr['AppBigType'] = $appbigtype;
	$appdetailinfo_arr['AppRanking'] = 0;
	$appdetailinfo_arr['AppRankingType'] = $catchtype;

	$appname = $html_detail->find('div.info-top dl.clearfix span[id=appname]',0)->plaintext;
	$appdetailinfo_arr['name'] = addslashes($appname);

	$scoredata = $html_detail->find('div.info-top dl.clearfix dd.info-star b.color-orange',0)->plaintext;
	$scoredata = str_replace("分","",$scoredata);
	$scoredata = str_replace(" ","",$scoredata);
	$appdetailinfo_arr['Rating'] = $scoredata;
	
	//评分人数
	$scoredata = $html_detail->find('div.info-top dl.clearfix dd.info-star span.color-gray',0)->plaintext;
	$scoredata = str_replace("已有","",$scoredata);
	$scoredata = str_replace("人评分","",$scoredata);
	$scoredata = str_replace("(","",$scoredata);
	$scoredata = str_replace(")","",$scoredata);
	$scoredata = str_replace(" ","",$scoredata);
	$appdetailinfo_arr['commentcount'] = $scoredata;
	
	$commentidstr = $html_detail->find('div.middle div.cls div.content-main div.content-main-border section.s-index-page',0)->outertext;
	$appdetailinfo_arr['commentid'] = getdatafromstring($commentidstr,"data-groupid");
	
	$html_detail->clear();
	return $appdetailinfo_arr;
}
function getdownloadtimes($downloadtimes)
{
	if(strstr($downloadtimes,"亿"))
	{
		$downloadtimes = str_replace("亿","",$downloadtimes) * 100000000;
	}
	if(strstr($downloadtimes,"万"))
	{
		$downloadtimes = str_replace("万","",$downloadtimes) * 10000;
	}
	if(strstr($downloadtimes,"千"))
	{
		$downloadtimes = str_replace("千","",$downloadtimes) * 1000;
	}
	if(strstr($downloadtimes,"百"))
	{
		$downloadtimes = str_replace("百","",$downloadtimes) * 100;
	}
	return $downloadtimes;
}


function get_exit_appinfo_fromdb_forcatch()
{
	$link = mysql_connect("localhost","root",DBPAS); 
	$createtime = strtotime(date("Y-m-d H:i:s"));
	$today = date("Y-m-d");
	$sql = 'select id, app_id, packageName,title,apptype,appbigtype from jk_apps_info_baidu where (apptype>=7 and apptype<=8) and IsDeleted=0 and id not in (select distinct(app_id) from jk_apps_score_data_mobile_baidu_'.date('Ym',time()).'  where time="'.$today.'")'; //
	$rs = mysql_query($sql, $link);
	
	$app_exit_info = array();
  	while($row=mysql_fetch_array($rs))        
  	{
  		$app_exit_info_temp = array();
  		$app_exit_info_temp['id'] = $row['id'];
  		$app_exit_info_temp['app_id'] = $row['app_id'];
	  	$app_exit_info_temp['AppPackageName'] = $row['packageName'];
	  	$app_exit_info_temp['AppName'] = $row['title'];
	  	$app_exit_info_temp['apptype'] = $row['apptype'];
	  	$app_exit_info_temp['appbigtype'] = $row['appbigtype'];
	  	$app_exit_info[] = $app_exit_info_temp;
  	}
	return $app_exit_info;
}



function writelog($rs,$id)
{return ;
	$filename_update = LOGPATHHEADER.'logs/test_1.log';
	$filename_noupdate = LOGPATHHEADER.'logs/test_11.log';
	$time = date("Y-m-d H:i:s");
	try
	{
		if($rs)
		{
			//记录错误，二次抓取用
			$info_str = $id."--".$time."\r\n";
			$fh = fopen($filename_update, "a");
			fwrite($fh, $info_str);
			fclose($fh);
		}
		else
		{
			$info_str = $id."--".$time."\r\n";
			$filename = $filename_noupdate;
			$fh = fopen($filename, "a");
			fwrite($fh, $info_str);
			fclose($fh);
		}
	}
	catch(Exception $e)
	{
		
	}
}


function writefailelog($rs,$id)
{return ;
	$filename_update = LOGPATHHEADER.'logs/baidufeilfal.log';
	$time = date("Y-m-d H:i:s");
	try
	{
		//记录错误，二次抓取用
		$info_str = $id."--".$time."\r\n";
		$fh = fopen($filename_update, "a");
		fwrite($fh, $info_str);
		fclose($fh);
	}
	catch(Exception $e)
	{
		
	}
}

?>